use datafusion::common::ScalarValue;
use datafusion::logical_expr::{Between, BinaryExpr, Expr, Operator};
use time::OffsetDateTime;
use crate::encode::schema::col;
use crate::planner::split::TimeRange;
pub fn time_range(filters: &[Expr]) -> TimeRange {
filters
.iter()
.map(range_of)
.fold(TimeRange::unbounded(), intersect)
}
pub fn range_filters(range: TimeRange) -> Vec<Expr> {
let mut out = Vec::with_capacity(2);
if let Some(start) = range.start() {
out.push(datafusion::logical_expr::col(col::FROM).gt_eq(timestamp_lit(start)));
}
if let Some(end) = range.end() {
out.push(datafusion::logical_expr::col(col::FROM).lt(timestamp_lit(end)));
}
out
}
pub fn version_ceiling(max: crate::version::Version) -> Expr {
use crate::encode::schema::{VERSION_PRECISION, VERSION_SCALE};
datafusion::logical_expr::col(col::VERSION).lt_eq(datafusion::logical_expr::lit(
ScalarValue::Decimal128(Some(max.to_i128()), VERSION_PRECISION, VERSION_SCALE),
))
}
pub fn recorded_at_ceiling(at: OffsetDateTime) -> Expr {
datafusion::logical_expr::col(col::RECORDED_AT).lt_eq(timestamp_lit(at))
}
fn timestamp_lit(t: OffsetDateTime) -> Expr {
datafusion::logical_expr::lit(crate::encode::schema::timestamp_scalar(t))
}
fn range_of(filter: &Expr) -> TimeRange {
match filter {
Expr::BinaryExpr(BinaryExpr { left, op, right }) => match op {
Operator::And => intersect(range_of(left), range_of(right)),
Operator::Or => TimeRange::unbounded(),
Operator::Lt | Operator::LtEq | Operator::Gt | Operator::GtEq | Operator::Eq => {
as_from_bound(left, *op, right).unwrap_or_else(TimeRange::unbounded)
}
_ => TimeRange::unbounded(),
},
Expr::Between(Between {
expr,
negated: false,
low,
high,
}) if is_from_column(expr) => {
match (as_timestamp(low), as_timestamp(high)) {
(Some(lo), Some(hi)) => TimeRange::new(Some(lo), Some(exclusive_after(hi))),
_ => TimeRange::unbounded(),
}
}
_ => TimeRange::unbounded(),
}
}
fn as_from_bound(left: &Expr, op: Operator, right: &Expr) -> Option<TimeRange> {
if is_from_column(left) {
bound_from(op, as_timestamp(right)?)
} else if is_from_column(right) {
bound_from(flip(op), as_timestamp(left)?)
} else {
None
}
}
fn bound_from(op: Operator, value: OffsetDateTime) -> Option<TimeRange> {
Some(match op {
Operator::Lt => TimeRange::new(None, Some(value)),
Operator::LtEq => TimeRange::new(None, Some(exclusive_after(value))),
Operator::Gt => TimeRange::new(Some(exclusive_after(value)), None),
Operator::GtEq => TimeRange::new(Some(value), None),
Operator::Eq => TimeRange::new(Some(value), Some(exclusive_after(value))),
_ => return None,
})
}
fn flip(op: Operator) -> Operator {
match op {
Operator::Lt => Operator::Gt,
Operator::LtEq => Operator::GtEq,
Operator::Gt => Operator::Lt,
Operator::GtEq => Operator::LtEq,
other => other,
}
}
fn exclusive_after(t: OffsetDateTime) -> OffsetDateTime {
t + time::Duration::microseconds(1)
}
fn is_from_column(expr: &Expr) -> bool {
match expr {
Expr::Column(c) => c.name == col::FROM,
Expr::Cast(cast) if is_lossless_target(&cast.data_type) => is_from_column(&cast.expr),
Expr::TryCast(cast) if is_lossless_target(&cast.data_type) => is_from_column(&cast.expr),
_ => false,
}
}
fn is_lossless_target(ty: &crate::arrow::datatypes::DataType) -> bool {
use crate::arrow::datatypes::{DataType, TimeUnit};
matches!(
ty,
DataType::Timestamp(TimeUnit::Microsecond | TimeUnit::Nanosecond, _)
)
}
fn as_timestamp(expr: &Expr) -> Option<OffsetDateTime> {
let scalar = match expr {
Expr::Literal(v, _) => v,
Expr::Cast(cast) if is_lossless_target(&cast.data_type) => {
return as_timestamp(&cast.expr);
}
Expr::TryCast(cast) if is_lossless_target(&cast.data_type) => {
return as_timestamp(&cast.expr);
}
_ => return None,
};
let nanos: i128 = match scalar {
ScalarValue::TimestampNanosecond(Some(v), _) => i128::from(*v),
ScalarValue::TimestampMicrosecond(Some(v), _) => i128::from(*v) * 1_000,
ScalarValue::TimestampMillisecond(Some(v), _) => i128::from(*v) * 1_000_000,
ScalarValue::TimestampSecond(Some(v), _) => i128::from(*v) * 1_000_000_000,
_ => return None,
};
OffsetDateTime::from_unix_timestamp_nanos(nanos).ok()
}
fn intersect(a: TimeRange, b: TimeRange) -> TimeRange {
let start = match (a.start(), b.start()) {
(Some(x), Some(y)) => Some(x.max(y)),
(Some(x), None) | (None, Some(x)) => Some(x),
(None, None) => None,
};
let end = match (a.end(), b.end()) {
(Some(x), Some(y)) => Some(x.min(y)),
(Some(x), None) | (None, Some(x)) => Some(x),
(None, None) => None,
};
TimeRange::new(start, end)
}
#[cfg(test)]
mod tests {
use super::*;
use datafusion::logical_expr::{col as df_col, lit};
use time::macros::datetime;
const T10: OffsetDateTime = datetime!(2026-07-10 00:00 UTC);
const T20: OffsetDateTime = datetime!(2026-07-20 00:00 UTC);
fn ts(t: OffsetDateTime) -> Expr {
lit(crate::encode::schema::timestamp_scalar(t))
}
fn from() -> Expr {
df_col(col::FROM)
}
#[test]
fn range_filters_round_trip_through_extraction() {
let range = TimeRange::between(T10, T20);
let recovered = time_range(&range_filters(range));
assert_eq!(recovered, range);
}
#[test]
fn range_filters_omit_absent_bounds() {
assert!(range_filters(TimeRange::unbounded()).is_empty());
assert_eq!(range_filters(TimeRange::new(Some(T10), None)).len(), 1);
assert_eq!(range_filters(TimeRange::new(None, Some(T20))).len(), 1);
assert_eq!(range_filters(TimeRange::between(T10, T20)).len(), 2);
}
#[test]
fn no_filters_gives_an_unbounded_range() {
assert_eq!(time_range(&[]), TimeRange::unbounded());
}
#[test]
fn greater_than_or_equal_becomes_an_inclusive_lower_bound() {
let r = time_range(&[from().gt_eq(ts(T10))]);
assert_eq!(r.start(), Some(T10));
assert_eq!(r.end(), None);
}
#[test]
fn strictly_greater_than_excludes_the_bound_itself() {
let r = time_range(&[from().gt(ts(T10))]);
assert_eq!(r.start(), Some(exclusive_after(T10)));
assert!(r.start().unwrap() > T10);
}
#[test]
fn less_than_becomes_an_exclusive_upper_bound() {
let r = time_range(&[from().lt(ts(T20))]);
assert_eq!(r.end(), Some(T20));
assert_eq!(r.start(), None);
}
#[test]
fn less_than_or_equal_still_matches_the_bound() {
let r = time_range(&[from().lt_eq(ts(T20))]);
assert!(r.end().unwrap() > T20);
}
#[test]
fn a_conjunction_intersects_bounds() {
let r = time_range(&[from().gt_eq(ts(T10)), from().lt(ts(T20))]);
assert_eq!(r, TimeRange::between(T10, T20));
}
#[test]
fn a_nested_and_intersects_too() {
let r = time_range(&[from().gt_eq(ts(T10)).and(from().lt(ts(T20)))]);
assert_eq!(r, TimeRange::between(T10, T20));
}
#[test]
fn the_tightest_bound_wins() {
let mid = datetime!(2026-07-15 00:00 UTC);
let r = time_range(&[from().gt_eq(ts(T10)), from().gt_eq(ts(mid))]);
assert_eq!(r.start(), Some(mid));
}
#[test]
fn a_literal_on_the_left_flips_the_operator() {
let r = time_range(&[ts(T10).lt_eq(from())]);
assert_eq!(r.start(), Some(T10));
assert_eq!(r.end(), None);
}
#[test]
fn between_is_inclusive_on_both_sides() {
let r = time_range(&[from().between(ts(T10), ts(T20))]);
assert_eq!(r.start(), Some(T10));
assert!(r.end().unwrap() > T20, "the high value must still match");
}
#[test]
fn equality_is_a_single_instant() {
let r = time_range(&[from().eq(ts(T10))]);
assert_eq!(r.start(), Some(T10));
assert!(r.end().unwrap() > T10);
assert!(!r.is_empty());
}
#[test]
fn a_disjunction_yields_no_bound() {
let r = time_range(&[from().gt_eq(ts(T10)).or(from().lt(ts(T20)))]);
assert_eq!(r, TimeRange::unbounded());
}
#[test]
fn a_disjunction_does_not_poison_a_sibling_conjunct() {
let r = time_range(&[
from().gt_eq(ts(T10)),
from().gt_eq(ts(T20)).or(from().lt(ts(T10))),
]);
assert_eq!(r.start(), Some(T10));
}
#[test]
fn predicates_on_other_columns_are_ignored() {
let r = time_range(&[df_col(col::MALO_ID).eq(lit("12345678905"))]);
assert_eq!(r, TimeRange::unbounded());
}
#[test]
fn a_bound_on_another_timestamp_column_is_not_used() {
let r = time_range(&[df_col(col::TO).gt_eq(ts(T10))]);
assert_eq!(r, TimeRange::unbounded());
}
fn cast_from(ty: crate::arrow::datatypes::DataType) -> Expr {
Expr::Cast(datafusion::logical_expr::Cast::new(Box::new(from()), ty))
}
fn timestamp(unit: crate::arrow::datatypes::TimeUnit) -> crate::arrow::datatypes::DataType {
crate::arrow::datatypes::DataType::Timestamp(unit, Some("UTC".into()))
}
#[test]
fn a_lossless_cast_around_the_column_is_seen_through() {
use crate::arrow::datatypes::TimeUnit;
for unit in [TimeUnit::Microsecond, TimeUnit::Nanosecond] {
let r = time_range(&[cast_from(timestamp(unit)).gt_eq(ts(T10))]);
assert_eq!(r.start(), Some(T10), "{unit:?}");
}
}
#[test]
fn a_truncating_cast_is_not_mistaken_for_a_bound_on_the_column() {
use crate::arrow::datatypes::{DataType, TimeUnit};
for ty in [
DataType::Date32,
timestamp(TimeUnit::Second),
timestamp(TimeUnit::Millisecond),
] {
assert_eq!(
time_range(&[cast_from(ty.clone()).eq(ts(T10))]),
TimeRange::unbounded(),
"{ty:?} truncates, so it cannot bound `from`"
);
}
}
#[test]
fn a_lossless_cast_over_a_truncating_one_is_still_refused() {
use crate::arrow::datatypes::{DataType, TimeUnit};
let inner = cast_from(DataType::Date32);
let outer = Expr::Cast(datafusion::logical_expr::Cast::new(
Box::new(inner),
timestamp(TimeUnit::Microsecond),
));
assert_eq!(time_range(&[outer.eq(ts(T10))]), TimeRange::unbounded());
}
#[test]
fn timestamp_literals_in_other_units_are_understood() {
let secs = lit(ScalarValue::TimestampSecond(
Some(T10.unix_timestamp()),
None,
));
assert_eq!(time_range(&[from().gt_eq(secs)]).start(), Some(T10));
let millis = lit(ScalarValue::TimestampMillisecond(
Some(T10.unix_timestamp() * 1_000),
None,
));
assert_eq!(time_range(&[from().gt_eq(millis)]).start(), Some(T10));
}
#[test]
fn a_truncating_cast_around_the_literal_yields_no_bound_either() {
use crate::arrow::datatypes::{DataType, TimeUnit};
let inner = ts(T10 + time::Duration::microseconds(500_000));
for ty in [
timestamp(TimeUnit::Second),
timestamp(TimeUnit::Millisecond),
DataType::Date32,
] {
let literal = Expr::Cast(datafusion::logical_expr::Cast::new(
Box::new(inner.clone()),
ty.clone(),
));
assert_eq!(
time_range(&[from().gt_eq(literal)]),
TimeRange::unbounded(),
"{ty:?} truncates the literal, so it cannot bound `from`"
);
}
}
#[test]
fn a_widening_cast_around_the_literal_is_still_seen_through() {
use crate::arrow::datatypes::TimeUnit;
for unit in [TimeUnit::Microsecond, TimeUnit::Nanosecond] {
let literal = Expr::Cast(datafusion::logical_expr::Cast::new(
Box::new(ts(T10)),
timestamp(unit),
));
assert_eq!(time_range(&[from().gt_eq(literal)]).start(), Some(T10));
}
}
#[test]
fn a_null_timestamp_yields_no_bound() {
let null = lit(ScalarValue::TimestampMicrosecond(None, None));
assert_eq!(time_range(&[from().gt_eq(null)]), TimeRange::unbounded());
}
#[test]
fn contradictory_bounds_produce_an_empty_range() {
let r = time_range(&[from().gt_eq(ts(T20)), from().lt(ts(T10))]);
assert!(r.is_empty());
}
#[test]
fn every_recognised_shape_still_narrows_the_range() {
for filter in [
from().gt_eq(ts(T10)),
from().lt(ts(T20)),
from().gt_eq(ts(T10)).and(from().lt(ts(T20))),
from().between(ts(T10), ts(T20)),
] {
assert_ne!(
time_range(std::slice::from_ref(&filter)),
TimeRange::unbounded(),
"{filter} must still narrow the scan"
);
}
}
#[test]
fn unrecognised_shapes_widen_rather_than_narrow() {
for filter in [
df_col(col::MALO_ID).eq(lit("x")),
from().gt_eq(ts(T10)).or(from().lt(ts(T20))),
from().is_null(),
] {
assert_eq!(
time_range(std::slice::from_ref(&filter)),
TimeRange::unbounded(),
"{filter} must not be mistaken for a bound"
);
}
}
}
#[cfg(test)]
mod properties {
use super::*;
use datafusion::logical_expr::{col as df_col, lit};
use proptest::prelude::*;
#[derive(Debug, Clone)]
enum Pred {
Cmp(Operator, i64),
Flipped(Operator, i64),
Between(i64, i64),
Foreign,
And(Box<Pred>, Box<Pred>),
Or(Box<Pred>, Box<Pred>),
}
fn at(seconds: i64) -> OffsetDateTime {
OffsetDateTime::from_unix_timestamp(seconds).expect("in range")
}
fn ts(seconds: i64) -> Expr {
lit(ScalarValue::TimestampMicrosecond(
Some(seconds * 1_000_000),
Some("UTC".into()),
))
}
impl Pred {
fn to_expr(&self) -> Expr {
let from = df_col(col::FROM);
match self {
Self::Cmp(op, t) => binary(from, *op, ts(*t)),
Self::Flipped(op, t) => binary(ts(*t), *op, from),
Self::Between(lo, hi) => from.between(ts(*lo), ts(*hi)),
Self::Foreign => df_col(col::MALO_ID).eq(lit("12345678905")),
Self::And(a, b) => a.to_expr().and(b.to_expr()),
Self::Or(a, b) => a.to_expr().or(b.to_expr()),
}
}
fn holds(&self, t: i64) -> bool {
match self {
Self::Cmp(op, v) => compare(t, *op, *v),
Self::Flipped(op, v) => compare(*v, *op, t),
Self::Between(lo, hi) => t >= *lo && t <= *hi,
Self::Foreign => true,
Self::And(a, b) => a.holds(t) && b.holds(t),
Self::Or(a, b) => a.holds(t) || b.holds(t),
}
}
}
fn binary(left: Expr, op: Operator, right: Expr) -> Expr {
Expr::BinaryExpr(BinaryExpr::new(Box::new(left), op, Box::new(right)))
}
fn compare(left: i64, op: Operator, right: i64) -> bool {
match op {
Operator::Lt => left < right,
Operator::LtEq => left <= right,
Operator::Gt => left > right,
Operator::GtEq => left >= right,
Operator::Eq => left == right,
other => unreachable!("generator produces no {other:?}"),
}
}
fn contains(range: TimeRange, t: i64) -> bool {
let at = at(t);
range.start().is_none_or(|s| at >= s) && range.end().is_none_or(|e| at < e)
}
const WINDOW: std::ops::Range<i64> = 0..40;
fn leaf() -> impl Strategy<Value = Pred> {
let op = prop_oneof![
Just(Operator::Lt),
Just(Operator::LtEq),
Just(Operator::Gt),
Just(Operator::GtEq),
Just(Operator::Eq),
];
prop_oneof![
8 => (op.clone(), WINDOW).prop_map(|(o, t)| Pred::Cmp(o, t)),
4 => (op, WINDOW).prop_map(|(o, t)| Pred::Flipped(o, t)),
3 => (WINDOW, WINDOW).prop_map(|(a, b)| Pred::Between(a.min(b), a.max(b))),
1 => Just(Pred::Foreign),
]
}
fn predicate() -> impl Strategy<Value = Pred> {
leaf().prop_recursive(4, 24, 2, |inner| {
prop_oneof![
inner
.clone()
.prop_flat_map(move |a| leaf()
.prop_map(move |b| Pred::And(Box::new(a.clone()), Box::new(b)))),
inner
.prop_flat_map(move |a| leaf()
.prop_map(move |b| Pred::Or(Box::new(a.clone()), Box::new(b)))),
]
})
}
proptest! {
#[test]
fn an_admitted_row_is_never_outside_the_extracted_range(
pred in predicate(),
probe in WINDOW,
) {
let range = time_range(&[pred.to_expr()]);
prop_assert!(
!pred.holds(probe) || contains(range, probe),
"{pred:?} admits {probe} but the extracted range {range:?} excludes it",
);
}
#[test]
fn intersecting_several_filters_stays_conservative(
preds in prop::collection::vec(predicate(), 1..4),
probe in WINDOW,
) {
let exprs: Vec<Expr> = preds.iter().map(Pred::to_expr).collect();
let range = time_range(&exprs);
let admitted = preds.iter().all(|p| p.holds(probe));
prop_assert!(
!admitted || contains(range, probe),
"{preds:?} admit {probe} but {range:?} excludes it",
);
}
#[test]
fn range_filters_re_extract_to_the_same_bounds(a in WINDOW, b in WINDOW) {
let range = TimeRange::between(at(a.min(b)), at(a.max(b) + 1));
prop_assert_eq!(time_range(&range_filters(range)), range);
}
}
}