use std::sync::Arc;
use arrow::array::{AsArray, TimestampNanosecondArray};
use arrow::datatypes::{DataType, TimeUnit, TimestampNanosecondType};
use datafusion::common::{Result as DFResult, ScalarValue};
use datafusion::logical_expr::{
ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility,
};
#[derive(Debug, PartialEq, Eq, Hash)]
pub(super) struct TimeBucketUdf {
signature: Signature,
}
impl TimeBucketUdf {
pub(super) fn new() -> Self {
Self {
signature: Signature::new(
TypeSignature::OneOf(vec![
TypeSignature::Exact(vec![
DataType::Utf8,
DataType::Timestamp(TimeUnit::Nanosecond, None),
]),
TypeSignature::Exact(vec![
DataType::Utf8,
DataType::Timestamp(TimeUnit::Nanosecond, None),
DataType::Utf8,
]),
]),
Volatility::Immutable,
),
}
}
}
impl ScalarUDFImpl for TimeBucketUdf {
fn name(&self) -> &'static str {
"time_bucket"
}
fn signature(&self) -> &Signature {
&self.signature
}
fn return_type(&self, _arg_types: &[DataType]) -> DFResult<DataType> {
Ok(DataType::Timestamp(TimeUnit::Nanosecond, None))
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DFResult<ColumnarValue> {
let args = args.args;
let interval_str = match &args[0] {
ColumnarValue::Scalar(ScalarValue::Utf8(Some(s))) => s.clone(),
_ => {
return Err(datafusion::common::DataFusionError::Plan(
"time_bucket: first argument must be a string interval like '5m'".into(),
))
}
};
let interval_nanos = parse_interval(&interval_str)?;
let tz: Option<chrono_tz::Tz> = if args.len() >= 3 {
match &args[2] {
ColumnarValue::Scalar(ScalarValue::Utf8(Some(tz_str))) => {
let parsed: chrono_tz::Tz = tz_str.parse().map_err(|_| {
datafusion::common::DataFusionError::Plan(format!(
"time_bucket: unknown timezone '{tz_str}'. Use IANA names like 'America/New_York'"
))
})?;
Some(parsed)
}
_ => None,
}
} else {
None
};
match &args[1] {
ColumnarValue::Array(arr) => {
let ts_array = arr.as_primitive::<TimestampNanosecondType>();
let result: TimestampNanosecondArray = ts_array
.iter()
.map(|opt: Option<i64>| {
opt.map(|ts| bucket_timestamp(ts, interval_nanos, tz.as_ref()))
})
.collect();
Ok(ColumnarValue::Array(Arc::new(result)))
}
ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(ts), tz_arc)) => {
let bucketed = bucket_timestamp(*ts, interval_nanos, tz.as_ref());
Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
Some(bucketed),
tz_arc.clone(),
)))
}
_ => Err(datafusion::common::DataFusionError::Plan(
"time_bucket: second argument must be a timestamp".into(),
)),
}
}
}
fn bucket_timestamp(ts_nanos: i64, interval_nanos: i64, tz: Option<&chrono_tz::Tz>) -> i64 {
let utc_truncated = ts_nanos - ts_nanos.rem_euclid(interval_nanos);
let Some(tz) = tz else {
return utc_truncated;
};
use chrono::{DateTime, TimeZone};
let utc_dt = DateTime::from_timestamp_nanos(ts_nanos);
let Some(local_nanos) = utc_dt
.with_timezone(tz)
.naive_local()
.and_utc()
.timestamp_nanos_opt()
else {
return utc_truncated;
};
let bucketed_local = local_nanos - local_nanos.rem_euclid(interval_nanos);
let bucketed_naive = DateTime::from_timestamp_nanos(bucketed_local).naive_utc();
match tz.from_local_datetime(&bucketed_naive) {
chrono::LocalResult::Single(dt) => dt.timestamp_nanos_opt().unwrap_or(utc_truncated),
chrono::LocalResult::Ambiguous(earliest, _) => {
earliest.timestamp_nanos_opt().unwrap_or(utc_truncated)
}
chrono::LocalResult::None => {
const MINUTE_NS: i64 = 60 * 1_000_000_000;
(1..=180)
.find_map(|step| {
let probe = DateTime::from_timestamp_nanos(
bucketed_local.saturating_add(step * MINUTE_NS),
)
.naive_utc();
match tz.from_local_datetime(&probe) {
chrono::LocalResult::Single(dt) => dt.timestamp_nanos_opt(),
chrono::LocalResult::Ambiguous(earliest, _) => {
earliest.timestamp_nanos_opt()
}
chrono::LocalResult::None => None,
}
})
.unwrap_or(utc_truncated)
}
}
}
pub(super) fn parse_interval(s: &str) -> DFResult<i64> {
let s = s.trim();
if s.is_empty() {
return Err(datafusion::common::DataFusionError::Plan(
"empty interval string".into(),
));
}
let (num_str, suffix) = s.split_at(s.find(|c: char| !c.is_ascii_digit()).unwrap_or(s.len()));
let num: i64 = num_str.parse().map_err(|_| {
datafusion::common::DataFusionError::Plan(format!("invalid interval number: '{num_str}'"))
})?;
let nanos_per_unit: i64 = match suffix.trim() {
"ns" => 1,
"us" | "µs" => 1_000,
"ms" => 1_000_000,
"s" => 1_000_000_000,
"m" | "min" => 60 * 1_000_000_000,
"h" => 3_600 * 1_000_000_000,
"d" => 86_400 * 1_000_000_000,
"w" => 7 * 86_400 * 1_000_000_000,
other => {
return Err(datafusion::common::DataFusionError::Plan(format!(
"unknown interval unit: '{other}'. Use ns, us, ms, s, m, h, d, w"
)))
}
};
let result = num.checked_mul(nanos_per_unit).ok_or_else(|| {
datafusion::common::DataFusionError::Plan(format!(
"interval overflow: {num} * {nanos_per_unit} exceeds i64 range"
))
})?;
if result <= 0 {
return Err(datafusion::common::DataFusionError::Plan(
"interval must be positive (e.g. '5m', '1h')".into(),
));
}
Ok(result)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_interval_various_units() {
assert_eq!(parse_interval("5m").unwrap(), 5 * 60 * 1_000_000_000);
assert_eq!(parse_interval("1h").unwrap(), 3_600 * 1_000_000_000);
assert_eq!(parse_interval("30s").unwrap(), 30 * 1_000_000_000);
assert_eq!(parse_interval("1d").unwrap(), 86_400 * 1_000_000_000);
assert_eq!(parse_interval("100ms").unwrap(), 100 * 1_000_000);
assert!(parse_interval("").is_err());
assert!(parse_interval("5x").is_err());
}
#[test]
fn parse_interval_rejects_zero() {
let err = parse_interval("0s").unwrap_err();
assert!(
err.to_string().contains("positive"),
"expected positive-interval error, got: {err}"
);
let err = parse_interval("0m").unwrap_err();
assert!(
err.to_string().contains("positive"),
"expected positive-interval error, got: {err}"
);
}
#[test]
fn time_bucket_udf() {
let udf = TimeBucketUdf::new();
let interval = ColumnarValue::Scalar(ScalarValue::Utf8(Some("1h".to_string())));
let ts = ColumnarValue::Array(Arc::new(arrow::array::TimestampNanosecondArray::from(
vec![
3_661_000_000_000i64, ],
)));
let result =
crate::sql::functions::helpers::invoke_udf(&udf, Vec::from([interval, ts]), 1).unwrap();
match &result {
ColumnarValue::Array(arr) => {
let ta = arr
.as_any()
.downcast_ref::<arrow::array::TimestampNanosecondArray>()
.unwrap();
assert_eq!(ta.value(0), 3_600_000_000_000);
}
_ => panic!("expected array"),
}
}
}
#[cfg(test)]
mod timezone_tests {
#![allow(clippy::unwrap_used, clippy::expect_used)]
use super::*;
const DAY: i64 = 86_400 * 1_000_000_000;
const HOUR: i64 = 3_600 * 1_000_000_000;
fn ts(s: &str) -> i64 {
chrono::DateTime::parse_from_rfc3339(s)
.unwrap()
.timestamp_nanos_opt()
.unwrap()
}
#[test]
fn a_zoned_day_bucket_is_local_midnight() {
let ny: chrono_tz::Tz = "America/New_York".parse().unwrap();
let got = bucket_timestamp(ts("2024-03-15T02:00:00Z"), DAY, Some(&ny));
assert_eq!(
got,
ts("2024-03-14T04:00:00Z"),
"22:00 local on the 14th belongs to the 14th's bucket"
);
let got = bucket_timestamp(ts("2024-03-15T18:00:00Z"), DAY, Some(&ny));
assert_eq!(got, ts("2024-03-15T04:00:00Z"));
}
#[test]
fn a_zoned_day_bucket_east_of_utc() {
let tokyo: chrono_tz::Tz = "Asia/Tokyo".parse().unwrap();
let got = bucket_timestamp(ts("2024-03-15T20:00:00Z"), DAY, Some(&tokyo));
assert_eq!(got, ts("2024-03-15T15:00:00Z"));
}
#[test]
fn dst_days_keep_their_local_midnights() {
let ny: chrono_tz::Tz = "America/New_York".parse().unwrap();
let before = bucket_timestamp(ts("2024-03-10T06:00:00Z"), DAY, Some(&ny)); let after = bucket_timestamp(ts("2024-03-10T20:00:00Z"), DAY, Some(&ny)); assert_eq!(before, ts("2024-03-10T05:00:00Z"), "local midnight EST");
assert_eq!(after, before, "both instants are the same local day");
let next = bucket_timestamp(ts("2024-03-11T12:00:00Z"), DAY, Some(&ny));
assert_eq!(next - before, 23 * HOUR, "a spring-forward day is 23 hours");
}
#[test]
fn an_unzoned_bucket_truncates_utc() {
assert_eq!(
bucket_timestamp(ts("2024-03-15T02:00:00Z"), DAY, None),
ts("2024-03-15T00:00:00Z")
);
assert_eq!(
bucket_timestamp(ts("2024-03-15T02:34:56Z"), HOUR, None),
ts("2024-03-15T02:00:00Z")
);
}
#[test]
fn a_half_hour_zone_buckets_on_its_own_clock() {
let kolkata: chrono_tz::Tz = "Asia/Kolkata".parse().unwrap(); let got = bucket_timestamp(ts("2024-03-15T00:10:00Z"), HOUR, Some(&kolkata));
assert_eq!(got, ts("2024-03-14T23:30:00Z"));
}
}