Skip to main content

vortex_array/scalar_fn/fns/
replace_time_zone.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Reinterpret timestamp wall times in a different timezone.
5
6use std::fmt;
7use std::sync::Arc;
8
9use jiff::Timestamp as JiffTimestamp;
10use jiff::tz::AmbiguousOffset;
11use jiff::tz::TimeZone;
12use prost::Message;
13use vortex_buffer::BitBufferMut;
14use vortex_buffer::BufferMut;
15use vortex_error::VortexError;
16use vortex_error::VortexResult;
17use vortex_error::vortex_bail;
18use vortex_error::vortex_err;
19use vortex_session::VortexSession;
20use vortex_session::registry::CachedId;
21
22use crate::ArrayRef;
23use crate::ExecutionCtx;
24use crate::IntoArray;
25use crate::arrays::ConstantArray;
26use crate::arrays::ExtensionArray;
27use crate::arrays::PrimitiveArray;
28use crate::arrays::VarBinViewArray;
29use crate::arrays::extension::ExtensionArrayExt;
30use crate::dtype::DType;
31use crate::dtype::Nullability;
32use crate::dtype::PType;
33use crate::expr::BoundExpression;
34use crate::expr::bound;
35use crate::extension::datetime::TimeUnit;
36use crate::extension::datetime::Timestamp;
37use crate::extension::datetime::TimestampOptions;
38use crate::extension::datetime::resolve_time_zone as resolve_zone;
39use crate::proto::expr as pb;
40use crate::scalar::Scalar;
41use crate::scalar_fn::Arity;
42use crate::scalar_fn::ChildName;
43use crate::scalar_fn::ExecutionArgs;
44use crate::scalar_fn::ScalarFnId;
45use crate::scalar_fn::ScalarFnVTable;
46use crate::scalar_fn::fns::literal::Literal;
47use crate::validity::Validity;
48
49/// Timezone replacement options. The ambiguity policy is a UTF-8 expression child.
50#[derive(Clone, Debug, PartialEq, Eq, Hash)]
51pub struct ReplaceTimeZoneOptions {
52    /// Destination timezone, or `None` to remove the timezone while preserving wall time.
53    pub time_zone: Option<Arc<str>>,
54    /// Return null for nonexistent wall times instead of raising an error.
55    pub null_on_non_existent: bool,
56}
57
58impl fmt::Display for ReplaceTimeZoneOptions {
59    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
60        match &self.time_zone {
61            Some(tz) => write!(f, "tz={tz}")?,
62            None => write!(f, "tz=none")?,
63        }
64        write!(f, ", null_on_non_existent={}", self.null_on_non_existent)
65    }
66}
67
68/// Replace a timestamp's timezone, preserving its local wall time.
69///
70/// The second child selects `raise`, `earliest`, `latest`, or `null` for ambiguous times.
71#[derive(Clone)]
72pub struct ReplaceTimeZone;
73
74impl ScalarFnVTable for ReplaceTimeZone {
75    type Options = ReplaceTimeZoneOptions;
76
77    fn id(&self) -> ScalarFnId {
78        static ID: CachedId = CachedId::new("vortex.replace_time_zone");
79        *ID
80    }
81
82    fn serialize(&self, options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
83        Ok(Some(
84            pb::ReplaceTimeZoneOpts {
85                time_zone: options.time_zone.as_ref().map(ToString::to_string),
86                null_on_non_existent: options.null_on_non_existent,
87            }
88            .encode_to_vec(),
89        ))
90    }
91
92    fn deserialize(
93        &self,
94        metadata: &[u8],
95        _session: &VortexSession,
96    ) -> VortexResult<Self::Options> {
97        let options = pb::ReplaceTimeZoneOpts::decode(metadata)?;
98        Ok(ReplaceTimeZoneOptions {
99            time_zone: options.time_zone.map(Arc::from),
100            null_on_non_existent: options.null_on_non_existent,
101        })
102    }
103
104    fn arity(&self, _options: &Self::Options) -> Arity {
105        Arity::Exact(2)
106    }
107
108    fn child_name(&self, _options: &Self::Options, child_idx: usize) -> ChildName {
109        match child_idx {
110            0 => ChildName::from("input"),
111            1 => ChildName::from("ambiguous"),
112            _ => unreachable!("Invalid child index {child_idx} for replace_time_zone()"),
113        }
114    }
115
116    fn simplify(
117        &self,
118        options: &Self::Options,
119        expr: &BoundExpression,
120    ) -> VortexResult<Option<BoundExpression>> {
121        let (Some(value), Some(policy)) = (
122            expr.child(0).as_opt::<Literal>(),
123            expr.child(1).as_opt::<Literal>(),
124        ) else {
125            return Ok(None);
126        };
127        // Fold to a literal so stats pruning sees a bare literal operand. A failing replacement
128        // (e.g. a raised ambiguity) is left in place so the error surfaces at execution time.
129        Ok(replace_scalar(options, value, policy).ok().map(bound::lit))
130    }
131
132    fn return_dtype(&self, options: &Self::Options, arg_dtypes: &[DType]) -> VortexResult<DType> {
133        let input = timestamp_options(&arg_dtypes[0])?;
134        if input.unit == TimeUnit::Days {
135            vortex_bail!("Timestamp cannot use day units");
136        }
137        if !matches!(arg_dtypes[1], DType::Utf8(_) | DType::Null) {
138            vortex_bail!(
139                "replace_time_zone() requires a UTF-8 ambiguity policy, got {}",
140                arg_dtypes[1]
141            );
142        }
143        resolve_zone(options.time_zone.as_deref())?;
144        // A row's ambiguity policy can request null even for non-nullable children.
145        Ok(DType::Extension(
146            Timestamp::new_with_tz(input.unit, options.time_zone.clone(), Nullability::Nullable)
147                .erased(),
148        ))
149    }
150
151    fn execute(
152        &self,
153        options: &Self::Options,
154        args: &dyn ExecutionArgs,
155        ctx: &mut ExecutionCtx,
156    ) -> VortexResult<ArrayRef> {
157        let input = args.get(0)?;
158        let ambiguous = args.get(1)?;
159        let dtype =
160            self.return_dtype(options, &[input.dtype().clone(), ambiguous.dtype().clone()])?;
161        let metadata = timestamp_options(input.dtype())?.clone();
162        let source = resolve_zone(metadata.tz.as_deref())?;
163        let target = resolve_zone(options.time_zone.as_deref())?;
164        let convert = |value, policy: &str| {
165            replace(
166                value,
167                metadata.unit,
168                &source,
169                &target,
170                policy,
171                options.null_on_non_existent,
172            )
173        };
174
175        if let (Some(value), Some(policy)) = (input.as_constant(), ambiguous.as_constant()) {
176            let scalar = replace_scalar(options, value, policy)?;
177            return Ok(ConstantArray::new(scalar, args.row_count()).into_array());
178        }
179
180        if matches!(ambiguous.dtype(), DType::Null) {
181            return Ok(ConstantArray::new(Scalar::null(dtype), args.row_count()).into_array());
182        }
183        let input = input.execute::<ExtensionArray>(ctx)?;
184        let storage = input
185            .storage_array()
186            .clone()
187            .execute::<PrimitiveArray>(ctx)?;
188        let values = storage.to_buffer::<i64>();
189        let valid = storage.validity()?.execute_mask(values.len(), ctx)?;
190        let policies = ambiguous.execute::<VarBinViewArray>(ctx)?;
191        let policies_valid = policies.validity()?.execute_mask(values.len(), ctx)?;
192        let mut output = BufferMut::<i64>::with_capacity(values.len());
193        let mut output_valid = BitBufferMut::with_capacity(values.len());
194        for (i, ((value, valid), policy_valid)) in values
195            .iter()
196            .zip(valid.iter())
197            .zip(policies_valid.iter())
198            .enumerate()
199        {
200            let replaced = if valid && policy_valid {
201                let bytes = policies.bytes_at(i);
202                // SAFETY: policies has Utf8 dtype and this row is valid.
203                let policy = unsafe { std::str::from_utf8_unchecked(&bytes) };
204                convert(*value, policy)?
205            } else {
206                None
207            };
208            output.push(replaced.unwrap_or_default());
209            output_valid.append(replaced.is_some());
210        }
211        let DType::Extension(ext) = dtype else {
212            unreachable!()
213        };
214        let storage = PrimitiveArray::new(
215            output.freeze(),
216            Validity::from_bit_buffer(output_valid.freeze(), Nullability::Nullable),
217        )
218        .into_array();
219        Ok(ExtensionArray::new(ext, storage).into_array())
220    }
221}
222
223/// Replace the timezone of a single timestamp scalar using a scalar ambiguity policy.
224fn replace_scalar(
225    options: &ReplaceTimeZoneOptions,
226    value: &Scalar,
227    policy: &Scalar,
228) -> VortexResult<Scalar> {
229    let DType::Extension(ext) =
230        ReplaceTimeZone.return_dtype(options, &[value.dtype().clone(), policy.dtype().clone()])?
231    else {
232        unreachable!("replace_time_zone() returns a timestamp")
233    };
234    let metadata = timestamp_options(value.dtype())?;
235    let result = if value.is_null() || policy.is_null() {
236        None
237    } else {
238        let storage = value.as_extension().to_storage_scalar();
239        let policy = policy
240            .as_utf8()
241            .value()
242            .ok_or_else(|| vortex_err!("Missing ambiguity policy"))?;
243        replace(
244            i64::try_from(&storage)?,
245            metadata.unit,
246            &resolve_zone(metadata.tz.as_deref())?,
247            &resolve_zone(options.time_zone.as_deref())?,
248            policy.as_str(),
249            options.null_on_non_existent,
250        )?
251    };
252    let storage = result.map_or_else(
253        || Scalar::null(DType::Primitive(PType::I64, Nullability::Nullable)),
254        |v| Scalar::primitive(v, Nullability::Nullable),
255    );
256    Scalar::try_extension_ref(ext, storage)
257}
258
259fn timestamp_options(dtype: &DType) -> VortexResult<&TimestampOptions> {
260    if let DType::Extension(ext) = dtype
261        && let Some(options) = ext.metadata_opt::<Timestamp>()
262    {
263        return Ok(options);
264    }
265    vortex_bail!("replace_time_zone() requires Timestamp, got {dtype}")
266}
267
268fn replace(
269    value: i64,
270    unit: TimeUnit,
271    source: &TimeZone,
272    target: &TimeZone,
273    policy: &str,
274    null_on_non_existent: bool,
275) -> VortexResult<Option<i64>> {
276    if !matches!(policy, "raise" | "earliest" | "latest" | "null") {
277        vortex_bail!(
278            "Invalid ambiguity policy {policy:?}: expected raise, earliest, latest, or null"
279        );
280    }
281    // Convert first so the same-timezone shortcut cannot pass values outside jiff's range through.
282    let timestamp = match unit {
283        TimeUnit::Seconds => JiffTimestamp::from_second(value),
284        TimeUnit::Milliseconds => JiffTimestamp::from_millisecond(value),
285        TimeUnit::Microseconds => JiffTimestamp::from_microsecond(value),
286        TimeUnit::Nanoseconds => JiffTimestamp::from_nanosecond(i128::from(value)),
287        TimeUnit::Days => vortex_bail!("Timestamp cannot use day units"),
288    }
289    .map_err(|error| {
290        VortexError::from(error)
291            .with_context(format!("replace_time_zone() cannot convert {value}{unit}"))
292    })?;
293    // Keeping the same timezone does not introduce a new ambiguity to reject.
294    if source == target && policy == "raise" {
295        return Ok(Some(value));
296    }
297    let wall_time = source.to_datetime(timestamp);
298    let ambiguous = target.to_ambiguous_timestamp(wall_time);
299    let timestamp = match ambiguous.offset() {
300        AmbiguousOffset::Unambiguous { .. } => ambiguous.unambiguous()?,
301        AmbiguousOffset::Gap { .. } if null_on_non_existent => return Ok(None),
302        AmbiguousOffset::Gap { .. } => ambiguous.unambiguous()?,
303        AmbiguousOffset::Fold { .. } => match policy {
304            "earliest" => ambiguous.earlier()?,
305            "latest" => ambiguous.later()?,
306            "null" => return Ok(None),
307            _ => ambiguous.unambiguous()?,
308        },
309    };
310    Ok(Some(match unit {
311        TimeUnit::Seconds => timestamp.as_second(),
312        TimeUnit::Milliseconds => timestamp.as_millisecond(),
313        TimeUnit::Microseconds => timestamp.as_microsecond(),
314        TimeUnit::Nanoseconds => i64::try_from(timestamp.as_nanosecond())
315            .map_err(|e| vortex_err!("Timestamp overflow: {e}"))?,
316        TimeUnit::Days => unreachable!(),
317    }))
318}
319
320#[cfg(test)]
321mod tests {
322    use std::error::Error;
323    use std::sync::LazyLock;
324
325    use rstest::rstest;
326    use vortex_error::VortexResult;
327    use vortex_error::vortex_err;
328    use vortex_session::VortexSession;
329
330    use super::ReplaceTimeZone;
331    use super::ReplaceTimeZoneOptions;
332    use super::replace;
333    use super::resolve_zone;
334    use crate::Canonical;
335    use crate::IntoArray;
336    use crate::VortexSessionExecute;
337    use crate::arrays::ExtensionArray;
338    use crate::arrays::PrimitiveArray;
339    use crate::arrays::StructArray;
340    use crate::arrays::VarBinViewArray;
341    use crate::assert_arrays_eq;
342    use crate::dtype::DType;
343    use crate::dtype::Nullability;
344    use crate::dtype::StructFields;
345    use crate::expr::get_item;
346    use crate::expr::lit;
347    use crate::expr::replace_time_zone;
348    use crate::expr::root;
349    use crate::extension::datetime::TimeUnit;
350    use crate::extension::datetime::Timestamp;
351    use crate::scalar::Scalar;
352    use crate::scalar_fn::ScalarFnVTable;
353    use crate::scalar_fn::fns::literal::Literal;
354
355    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
356
357    #[rstest]
358    #[case(TimeUnit::Seconds, 1)]
359    #[case(TimeUnit::Milliseconds, 1_000)]
360    #[case(TimeUnit::Microseconds, 1_000_000)]
361    #[case(TimeUnit::Nanoseconds, 1_000_000_000)]
362    fn preserves_units_and_fractional_values(
363        #[case] unit: TimeUnit,
364        #[case] scale: i64,
365    ) -> VortexResult<()> {
366        let utc = resolve_zone(None)?;
367        let new_york = resolve_zone(Some("America/New_York"))?;
368        let value = 1_705_320_000 * scale + scale / 2;
369        assert_eq!(
370            replace(value, unit, &utc, &new_york, "raise", false)?,
371            Some(value + 18_000 * scale)
372        );
373        assert_eq!(
374            replace(
375                value + 18_000 * scale,
376                unit,
377                &new_york,
378                &utc,
379                "raise",
380                false
381            )?,
382            Some(value)
383        );
384        Ok(())
385    }
386
387    #[test]
388    fn resolves_folds_and_gaps_separately() -> VortexResult<()> {
389        let utc = resolve_zone(None)?;
390        let new_york = resolve_zone(Some("America/New_York"))?;
391        let fold = "2024-11-03T01:30:00Z"
392            .parse::<jiff::Timestamp>()?
393            .as_second();
394        let gap = "2024-03-10T02:30:00Z"
395            .parse::<jiff::Timestamp>()?
396            .as_second();
397        let convert = |value, policy, null_on_non_existent| {
398            replace(
399                value,
400                TimeUnit::Seconds,
401                &utc,
402                &new_york,
403                policy,
404                null_on_non_existent,
405            )
406        };
407        assert_eq!(convert(fold, "earliest", false)?, Some(fold + 14_400));
408        assert_eq!(convert(fold, "latest", false)?, Some(fold + 18_000));
409        assert_eq!(convert(fold, "null", false)?, None);
410        assert!(convert(fold, "raise", false).is_err());
411        assert_eq!(convert(gap, "raise", true)?, None);
412        // Ambiguity policies must not resolve nonexistent wall times.
413        assert!(convert(gap, "earliest", false).is_err());
414        Ok(())
415    }
416
417    #[test]
418    fn rejects_invalid_policy_and_timestamp_overflow() -> VortexResult<()> {
419        let utc = resolve_zone(None)?;
420        assert!(replace(0, TimeUnit::Seconds, &utc, &utc, "invalid", false).is_err());
421        let new_york = resolve_zone(Some("America/New_York"))?;
422        assert!(
423            replace(
424                i64::MAX,
425                TimeUnit::Nanoseconds,
426                &utc,
427                &new_york,
428                "raise",
429                false
430            )
431            .is_err()
432        );
433        assert!(resolve_zone(Some("Not/A/Timezone")).is_err());
434        assert!(resolve_zone(Some("+99:00")).is_err());
435        assert!(resolve_zone(Some("-05:60")).is_err());
436        Ok(())
437    }
438
439    #[rstest]
440    #[case(None, false)]
441    #[case(Some("America/New_York"), true)]
442    fn options_round_trip(
443        #[case] time_zone: Option<&str>,
444        #[case] null_on_non_existent: bool,
445    ) -> VortexResult<()> {
446        let options = ReplaceTimeZoneOptions {
447            time_zone: time_zone.map(Into::into),
448            null_on_non_existent,
449        };
450        let metadata = ReplaceTimeZone.serialize(&options)?.unwrap();
451        assert_eq!(ReplaceTimeZone.deserialize(&metadata, &SESSION)?, options);
452        Ok(())
453    }
454
455    #[rstest]
456    #[case(None, Some("UTC"), 0)]
457    #[case(Some("+01:00"), None, 3_600)]
458    #[case(Some("-05:30"), None, -19_800)]
459    #[case(None, Some("+05:30"), -19_800)]
460    #[case(Some("+01:00"), Some("-05:30"), 23_400)]
461    fn simplify_folds_literal_input(
462        #[case] source: Option<&str>,
463        #[case] target: Option<&str>,
464        #[case] offset_seconds: i64,
465    ) -> VortexResult<()> {
466        let timestamp = |tz: Option<&str>| {
467            Timestamp::new_with_tz(
468                TimeUnit::Microseconds,
469                tz.map(Into::into),
470                Nullability::Nullable,
471            )
472            .erased()
473        };
474        let micros = Scalar::primitive(1_704_153_600_000_000i64, Nullability::Nullable);
475        let expr = replace_time_zone(
476            lit(Scalar::extension_ref(timestamp(source), micros)),
477            lit("earliest"),
478            ReplaceTimeZoneOptions {
479                time_zone: target.map(Into::into),
480                null_on_non_existent: false,
481            },
482        );
483        let scope = DType::Struct(StructFields::empty(), Nullability::NonNullable);
484        let optimized = expr.bind(&scope)?.optimize()?;
485
486        // Stats pruning only matches bare literal operands, so the replacement must fold.
487        let scalar = optimized
488            .as_opt::<Literal>()
489            .ok_or_else(|| vortex_err!("expected a bare literal, got {optimized}"))?;
490        assert_eq!(
491            scalar,
492            &Scalar::extension_ref(
493                timestamp(target),
494                Scalar::primitive(
495                    1_704_153_600_000_000i64 + offset_seconds * 1_000_000,
496                    Nullability::Nullable,
497                ),
498            )
499        );
500        assert_eq!(scalar.dtype(), &DType::Extension(timestamp(target)));
501        Ok(())
502    }
503
504    #[test]
505    fn applies_per_row_policies_and_nulls() -> VortexResult<()> {
506        let timestamp = |tz: Option<&str>| {
507            Timestamp::new_with_tz(
508                TimeUnit::Microseconds,
509                tz.map(Into::into),
510                Nullability::Nullable,
511            )
512            .erased()
513        };
514        // 2024-11-03 01:30 occurs twice in New York.
515        let fold = 1_730_597_400_000_000i64;
516        let hour = 3_600_000_000i64;
517        let input = ExtensionArray::new(
518            timestamp(None),
519            PrimitiveArray::from_option_iter([
520                Some(fold),
521                Some(fold),
522                Some(fold),
523                Some(fold),
524                None,
525            ])
526            .into_array(),
527        );
528        let policies = VarBinViewArray::from_iter_nullable_str([
529            Some("earliest"),
530            Some("latest"),
531            Some("null"),
532            None,
533            Some("raise"),
534        ]);
535        let array =
536            StructArray::from_fields(&[("t", input.into_array()), ("p", policies.into_array())])?
537                .into_array();
538        let result = array.apply(&replace_time_zone(
539            get_item("t", root()),
540            get_item("p", root()),
541            ReplaceTimeZoneOptions {
542                time_zone: Some("America/New_York".into()),
543                null_on_non_existent: false,
544            },
545        ))?;
546
547        let expected = ExtensionArray::new(
548            timestamp(Some("America/New_York")),
549            PrimitiveArray::from_option_iter([
550                Some(fold + 4 * hour),
551                Some(fold + 5 * hour),
552                None,
553                None,
554                None,
555            ])
556            .into_array(),
557        );
558        let mut ctx = SESSION.create_execution_ctx();
559        assert_arrays_eq!(result, expected, &mut ctx);
560        Ok(())
561    }
562
563    #[rstest]
564    #[case::end_of_9999(TimeUnit::Microseconds, 253_402_214_400_000_000)]
565    #[case::storage_min(TimeUnit::Seconds, i64::MIN)]
566    fn rejects_timestamps_beyond_jiff_range(
567        #[case] unit: TimeUnit,
568        #[case] value: i64,
569        #[values(None, Some("America/New_York"))] target: Option<&str>,
570    ) -> VortexResult<()> {
571        let input = ExtensionArray::new(
572            Timestamp::new(unit, Nullability::Nullable).erased(),
573            PrimitiveArray::from_option_iter([Some(0i64), Some(value)]).into_array(),
574        );
575        let expr = replace_time_zone(
576            root(),
577            lit("raise"),
578            ReplaceTimeZoneOptions {
579                time_zone: target.map(Into::into),
580                null_on_non_existent: false,
581            },
582        );
583        let mut ctx = SESSION.create_execution_ctx();
584        let result = input
585            .into_array()
586            .apply(&expr)
587            .and_then(|array| array.execute::<Canonical>(&mut ctx));
588        let error = result.unwrap_err();
589        assert!(error.to_string().contains("cannot convert"), "{error}");
590        assert!(
591            error
592                .source()
593                .is_some_and(|source| source.is::<jiff::Error>())
594        );
595
596        let control = ExtensionArray::new(
597            Timestamp::new(unit, Nullability::Nullable).erased(),
598            PrimitiveArray::from_option_iter([Some(0i64)]).into_array(),
599        );
600        control
601            .into_array()
602            .apply(&expr)?
603            .execute::<Canonical>(&mut ctx)?;
604
605        Ok(())
606    }
607}