use std::ops::{Add, BitAnd, BitOr, Div, Mul, Not, Rem, Sub};
use spark_connect_proto as proto;
use crate::expression::{
Alias, CaseWhen, Cast, CastEvalMode, ColumnReference, Expression, ExtractValue, FrameBoundary,
LiteralExpression, SortOrder, UnresolvedFunction, UpdateFieldsExpr, WindowExpressionWrapper,
};
use crate::types::DataType;
use crate::window::WindowSpec;
#[derive(Debug, Clone, PartialEq)]
pub struct Column {
expr: Expression,
}
impl Column {
pub fn new(expr: Expression) -> Self {
Column { expr }
}
pub fn expression(&self) -> &Expression {
&self.expr
}
pub fn alias(self, name: &str) -> Column {
Column {
expr: Expression::Alias(Box::new(Alias::new(self.expr, name))),
}
}
pub fn alias_with_metadata(
self,
name: &str,
metadata: std::collections::BTreeMap<String, String>,
) -> Column {
let json = serde_json::to_string(&metadata).unwrap_or_else(|_| "{}".to_string());
Column {
expr: Expression::Alias(Box::new(Alias::new(self.expr, name).with_metadata(json))),
}
}
pub fn name(self, name: &str) -> Column {
self.alias(name)
}
pub fn cast(self, to_type: DataType) -> Column {
Column {
expr: Expression::Cast(Box::new(Cast::new(self.expr, to_type))),
}
}
pub fn astype(self, to_type: DataType) -> Column {
self.cast(to_type)
}
pub fn cast_str(self, type_name: &str) -> Column {
Column {
expr: Expression::Cast(Box::new(Cast::new_str(self.expr, type_name))),
}
}
pub fn try_cast(self, to_type: DataType) -> Column {
Column {
expr: Expression::Cast(Box::new(
Cast::new(self.expr, to_type).with_eval_mode(CastEvalMode::Try),
)),
}
}
pub fn try_cast_str(self, type_name: &str) -> Column {
Column {
expr: Expression::Cast(Box::new(
Cast::new_str(self.expr, type_name).with_eval_mode(CastEvalMode::Try),
)),
}
}
pub fn is_null(self) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"isNull",
vec![self.expr],
)),
}
}
pub fn is_not_null(self) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"isNotNull",
vec![self.expr],
)),
}
}
pub fn substr(self, start: Column, length: Column) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"substr",
vec![self.expr, start.expr, length.expr],
)),
}
}
pub fn like(self, pattern: &str) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"like",
vec![
self.expr,
Expression::Literal(LiteralExpression::string(pattern)),
],
)),
}
}
pub fn rlike(self, pattern: &str) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"rlike",
vec![
self.expr,
Expression::Literal(LiteralExpression::string(pattern)),
],
)),
}
}
pub fn contains(self, other: Column) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"contains",
vec![self.expr, other.expr],
)),
}
}
pub fn ilike(self, pattern: &str) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"ilike",
vec![
self.expr,
Expression::Literal(LiteralExpression::string(pattern)),
],
)),
}
}
pub fn is_nan(self) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new("isNaN", vec![self.expr])),
}
}
pub fn eq_null_safe(self, other: Column) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"<=>",
vec![self.expr, other.expr],
)),
}
}
pub fn bitwise_and(self, other: Column) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"&",
vec![self.expr, other.expr],
)),
}
}
pub fn bitwise_or(self, other: Column) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"|",
vec![self.expr, other.expr],
)),
}
}
pub fn bitwise_xor(self, other: Column) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"^",
vec![self.expr, other.expr],
)),
}
}
pub fn between(self, lower: Column, upper: Column) -> Column {
let lo = Expression::UnresolvedFunction(UnresolvedFunction::new(
">=",
vec![self.expr.clone(), lower.expr],
));
let hi = Expression::UnresolvedFunction(UnresolvedFunction::new(
"<=",
vec![self.expr, upper.expr],
));
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new("and", vec![lo, hi])),
}
}
pub fn isin<C: Into<Column>>(self, values: impl IntoIterator<Item = C>) -> Column {
let values: Vec<Column> = values.into_iter().map(Into::into).collect();
let mut args = Vec::with_capacity(values.len() + 1);
args.push(self.expr);
args.extend(values.into_iter().map(|c| c.expr));
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new("in", args)),
}
}
pub fn with_field(self, field_name: &str, value: Column) -> Column {
Column {
expr: Expression::UpdateFields(Box::new(UpdateFieldsExpr::new(
self.expr,
field_name,
Some(value.expr),
))),
}
}
pub fn drop_fields(self, field_names: Vec<&str>) -> Column {
let mut expr = self.expr;
for name in field_names {
expr = Expression::UpdateFields(Box::new(UpdateFieldsExpr::new(expr, name, None)));
}
Column { expr }
}
pub fn startswith(self, other: Column) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"startsWith",
vec![self.expr, other.expr],
)),
}
}
pub fn endswith(self, other: Column) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"endsWith",
vec![self.expr, other.expr],
)),
}
}
pub fn asc(self) -> Column {
self.asc_nulls_first()
}
pub fn asc_nulls_first(self) -> Column {
Column {
expr: Expression::SortOrder(Box::new(SortOrder::asc_nulls_first(self.expr))),
}
}
pub fn asc_nulls_last(self) -> Column {
Column {
expr: Expression::SortOrder(Box::new(SortOrder::asc_nulls_last(self.expr))),
}
}
pub fn desc(self) -> Column {
self.desc_nulls_last()
}
pub fn desc_nulls_first(self) -> Column {
Column {
expr: Expression::SortOrder(Box::new(SortOrder::desc_nulls_first(self.expr))),
}
}
pub fn desc_nulls_last(self) -> Column {
Column {
expr: Expression::SortOrder(Box::new(SortOrder::desc_nulls_last(self.expr))),
}
}
pub fn when(self, condition: Column, value: Column) -> Column {
if let Expression::CaseWhen(case_when) = self.expr {
let mut branches = case_when.branches.clone();
branches.push((condition.expr, value.expr));
Column {
expr: Expression::CaseWhen(Box::new(CaseWhen {
branches,
else_expr: case_when.else_expr.clone(),
})),
}
} else {
Column {
expr: Expression::CaseWhen(Box::new(CaseWhen {
branches: vec![(condition.expr, value.expr)],
else_expr: None,
})),
}
}
}
pub fn otherwise(self, value: Column) -> Column {
if let Expression::CaseWhen(case_when) = self.expr {
Column {
expr: Expression::CaseWhen(Box::new(CaseWhen {
branches: case_when.branches.clone(),
else_expr: Some(Box::new(value.expr)),
})),
}
} else {
Column { expr: self.expr }
}
}
pub fn get_field(self, name: &str) -> Column {
let extraction = Expression::Literal(LiteralExpression::string(name));
Column {
expr: Expression::UnresolvedExtractValue(Box::new(ExtractValue::new(
self.expr, extraction,
))),
}
}
pub fn get_item(self, key: Column) -> Column {
Column {
expr: Expression::UnresolvedExtractValue(Box::new(ExtractValue::new(
self.expr, key.expr,
))),
}
}
pub fn to_proto(&self) -> proto::Expression {
self.expr.to_proto()
}
pub fn over(self, window_spec: WindowSpec) -> Column {
let frame_spec = window_spec.frame_spec.map(|(frame_type, lower, upper)| {
let frame_type_val = match frame_type {
crate::window::FrameType::Row => 1u32,
crate::window::FrameType::Range => 2u32,
};
let lower_boundary = match lower {
crate::window::FrameBound::UnboundedPreceding => FrameBoundary::UnboundedPreceding,
crate::window::FrameBound::Preceding(n) => FrameBoundary::Preceding(n),
crate::window::FrameBound::CurrentRow => FrameBoundary::CurrentRow,
crate::window::FrameBound::Following(n) => FrameBoundary::Following(n),
crate::window::FrameBound::UnboundedFollowing => FrameBoundary::UnboundedFollowing,
};
let upper_boundary = match upper {
crate::window::FrameBound::UnboundedPreceding => FrameBoundary::UnboundedPreceding,
crate::window::FrameBound::Preceding(n) => FrameBoundary::Preceding(n),
crate::window::FrameBound::CurrentRow => FrameBoundary::CurrentRow,
crate::window::FrameBound::Following(n) => FrameBoundary::Following(n),
crate::window::FrameBound::UnboundedFollowing => FrameBoundary::UnboundedFollowing,
};
(frame_type_val, lower_boundary, upper_boundary)
});
let window_expr = WindowExpressionWrapper::new(
self.expr,
window_spec.partition_spec,
window_spec.order_spec,
frame_spec,
);
Column {
expr: Expression::WindowExpression(Box::new(window_expr)),
}
}
pub fn eq(self, other: Column) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"==",
vec![self.expr, other.expr],
)),
}
}
pub fn ne(self, other: Column) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"not",
vec![Expression::UnresolvedFunction(UnresolvedFunction::new(
"==",
vec![self.expr, other.expr],
))],
)),
}
}
pub fn gt(self, other: Column) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
">",
vec![self.expr, other.expr],
)),
}
}
pub fn lt(self, other: Column) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"<",
vec![self.expr, other.expr],
)),
}
}
pub fn ge(self, other: Column) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
">=",
vec![self.expr, other.expr],
)),
}
}
pub fn le(self, other: Column) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"<=",
vec![self.expr, other.expr],
)),
}
}
pub fn add(self, other: Column) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"+",
vec![self.expr, other.expr],
)),
}
}
pub fn sub(self, other: Column) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"-",
vec![self.expr, other.expr],
)),
}
}
pub fn mul(self, other: Column) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"*",
vec![self.expr, other.expr],
)),
}
}
pub fn div(self, other: Column) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"/",
vec![self.expr, other.expr],
)),
}
}
pub fn modulo(self, other: Column) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"%",
vec![self.expr, other.expr],
)),
}
}
pub fn and(self, other: Column) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"and",
vec![self.expr, other.expr],
)),
}
}
pub fn or(self, other: Column) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"or",
vec![self.expr, other.expr],
)),
}
}
pub fn not(self) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new("not", vec![self.expr])),
}
}
pub fn neg(self) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new(
"negative",
vec![self.expr],
)),
}
}
}
impl Add for Column {
type Output = Column;
fn add(self, other: Column) -> Column {
self.add(other)
}
}
impl Sub for Column {
type Output = Column;
fn sub(self, other: Column) -> Column {
self.sub(other)
}
}
impl Mul for Column {
type Output = Column;
fn mul(self, other: Column) -> Column {
self.mul(other)
}
}
impl Div for Column {
type Output = Column;
fn div(self, other: Column) -> Column {
self.div(other)
}
}
impl Rem for Column {
type Output = Column;
fn rem(self, other: Column) -> Column {
self.modulo(other)
}
}
impl BitAnd for Column {
type Output = Column;
fn bitand(self, other: Column) -> Column {
self.and(other)
}
}
impl BitOr for Column {
type Output = Column;
fn bitor(self, other: Column) -> Column {
self.or(other)
}
}
impl Not for Column {
type Output = Column;
fn not(self) -> Column {
Column {
expr: Expression::UnresolvedFunction(UnresolvedFunction::new("not", vec![self.expr])),
}
}
}
pub fn col(name: &str) -> Column {
Column {
expr: Expression::ColumnReference(ColumnReference::new(name)),
}
}
impl From<&str> for Column {
fn from(name: &str) -> Column {
col(name)
}
}
impl From<String> for Column {
fn from(name: String) -> Column {
col(&name)
}
}
impl From<&String> for Column {
fn from(name: &String) -> Column {
col(name)
}
}
pub fn lit(value: i64) -> Column {
let lit = if i32::try_from(value).is_ok() {
LiteralExpression::int(value as i32)
} else {
LiteralExpression::long(value)
};
Column {
expr: Expression::Literal(lit),
}
}
pub fn lit_string(value: &str) -> Column {
Column {
expr: Expression::Literal(LiteralExpression::string(value)),
}
}
pub fn lit_double(value: f64) -> Column {
Column {
expr: Expression::Literal(LiteralExpression::double(value)),
}
}
pub fn lit_boolean(value: bool) -> Column {
Column {
expr: Expression::Literal(LiteralExpression::boolean(value)),
}
}
pub fn when(condition: Column, value: Column) -> Column {
Column {
expr: Expression::CaseWhen(Box::new(CaseWhen::new(vec![(condition.expr, value.expr)]))),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_col_creation() {
let c = col("x");
assert!(matches!(c.expr, Expression::ColumnReference(_)));
}
#[test]
fn test_lit_creation() {
let c = lit(42);
assert!(matches!(c.expr, Expression::Literal(_)));
}
#[test]
fn test_addition() {
let c1 = col("a");
let c2 = lit(1);
let result = c1.add(c2);
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
}
#[test]
fn test_alias() {
let c = col("x");
let aliased = c.alias("y");
assert!(matches!(aliased.expr, Expression::Alias(_)));
}
#[test]
fn test_cast() {
let c = col("x");
let casted = c.cast(DataType::String {
collation: "UTF8_BINARY".to_string(),
});
assert!(matches!(casted.expr, Expression::Cast(_)));
}
#[test]
fn test_comparison_operators() {
let c1 = col("a");
let c2 = col("b");
let result = c1.clone().eq(c2.clone());
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a").ne(col("b"));
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a").gt(col("b"));
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a").lt(col("b"));
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a").ge(col("b"));
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a").le(col("b"));
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
}
#[test]
fn test_arithmetic_operators() {
let result = col("a").sub(col("b"));
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a").mul(col("b"));
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a").div(col("b"));
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a").modulo(col("b"));
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
}
#[test]
fn test_logical_operators() {
let result = col("a").and(col("b"));
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a").or(col("b"));
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a").not();
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a").neg();
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
}
#[test]
fn test_operator_traits() {
let c1 = col("a");
let c2 = col("b");
let result = c1.clone() + c2.clone();
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a") - col("b");
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a") * col("b");
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a") / col("b");
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a") % col("b");
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a") & col("b");
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a") | col("b");
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = !col("a");
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
}
#[test]
fn test_from_implementations() {
let c1: Column = "x".into();
assert!(matches!(c1.expr, Expression::ColumnReference(_)));
let c2: Column = "y".to_string().into();
assert!(matches!(c2.expr, Expression::ColumnReference(_)));
let s = "z".to_string();
let c3: Column = (&s).into();
assert!(matches!(c3.expr, Expression::ColumnReference(_)));
}
#[test]
fn test_lit_functions() {
let c = lit(42);
assert!(matches!(
c.expr,
Expression::Literal(LiteralExpression::Integer(_))
));
let c = lit(5_000_000_000i64);
assert!(matches!(
c.expr,
Expression::Literal(LiteralExpression::Long(_))
));
let c = lit_string("hello");
assert!(matches!(
c.expr,
Expression::Literal(LiteralExpression::String(_))
));
let c = lit_double(3.14);
assert!(matches!(
c.expr,
Expression::Literal(LiteralExpression::Double(_))
));
let c = lit_boolean(true);
assert!(matches!(
c.expr,
Expression::Literal(LiteralExpression::Boolean(_))
));
}
#[test]
fn test_when_otherwise() {
let cond = col("x").gt(lit(5));
let result = when(cond, lit(1));
assert!(matches!(result.expr, Expression::CaseWhen(_)));
let cond2 = col("y").lt(lit(10));
let result2 = result.when(cond2, lit(2));
assert!(matches!(result2.expr, Expression::CaseWhen(_)));
let final_result = result2.otherwise(lit(99));
assert!(matches!(final_result.expr, Expression::CaseWhen(_)));
}
#[test]
fn test_is_null_and_is_not_null() {
let result = col("a").is_null();
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a").is_not_null();
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
}
#[test]
fn test_getitem() {
let result = col("array").get_item(lit(0));
assert!(matches!(result.expr, Expression::UnresolvedExtractValue(_)));
}
#[test]
fn test_between() {
let result = col("a").between(lit(1), lit(10));
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
}
#[test]
fn test_substring() {
let result = col("a").substr(lit(1), lit(3));
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
}
#[test]
fn test_various_methods() {
let result = col("a").cast_str("int");
assert!(matches!(result.expr, Expression::Cast(_)));
let result = col("a").desc();
assert!(matches!(result.expr, Expression::SortOrder(_)));
let result = col("a").desc_nulls_first();
assert!(matches!(result.expr, Expression::SortOrder(_)));
let result = col("a").desc_nulls_last();
assert!(matches!(result.expr, Expression::SortOrder(_)));
let result = col("a").asc();
assert!(matches!(result.expr, Expression::SortOrder(_)));
let result = col("a").asc_nulls_first();
assert!(matches!(result.expr, Expression::SortOrder(_)));
let result = col("a").asc_nulls_last();
assert!(matches!(result.expr, Expression::SortOrder(_)));
}
#[test]
fn test_get_field() {
let result = col("struct").get_field("field_name");
assert!(matches!(result.expr, Expression::UnresolvedExtractValue(_)));
}
#[test]
fn test_string_functions() {
let result = col("a").contains(col("b"));
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a").startswith(col("b"));
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a").endswith(col("b"));
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a").like("%pattern%");
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
let result = col("a").rlike("[0-9]+");
assert!(matches!(result.expr, Expression::UnresolvedFunction(_)));
}
}