1use std::fmt;
7use std::sync::Arc;
8
9use jiff::Timestamp as JiffTimestamp;
10use jiff::tz::AmbiguousOffset;
11use jiff::tz::TimeZone;
12use prost::Message;
13use vortex_buffer::BitBufferMut;
14use vortex_buffer::BufferMut;
15use vortex_error::VortexError;
16use vortex_error::VortexResult;
17use vortex_error::vortex_bail;
18use vortex_error::vortex_err;
19use vortex_session::VortexSession;
20use vortex_session::registry::CachedId;
21
22use crate::ArrayRef;
23use crate::ExecutionCtx;
24use crate::IntoArray;
25use crate::arrays::ConstantArray;
26use crate::arrays::ExtensionArray;
27use crate::arrays::PrimitiveArray;
28use crate::arrays::VarBinViewArray;
29use crate::arrays::extension::ExtensionArrayExt;
30use crate::dtype::DType;
31use crate::dtype::Nullability;
32use crate::dtype::PType;
33use crate::expr::BoundExpression;
34use crate::expr::bound;
35use crate::extension::datetime::TimeUnit;
36use crate::extension::datetime::Timestamp;
37use crate::extension::datetime::TimestampOptions;
38use crate::extension::datetime::resolve_time_zone as resolve_zone;
39use crate::proto::expr as pb;
40use crate::scalar::Scalar;
41use crate::scalar_fn::Arity;
42use crate::scalar_fn::ChildName;
43use crate::scalar_fn::ExecutionArgs;
44use crate::scalar_fn::ScalarFnId;
45use crate::scalar_fn::ScalarFnVTable;
46use crate::scalar_fn::fns::literal::Literal;
47use crate::validity::Validity;
48
49#[derive(Clone, Debug, PartialEq, Eq, Hash)]
51pub struct ReplaceTimeZoneOptions {
52 pub time_zone: Option<Arc<str>>,
54 pub null_on_non_existent: bool,
56}
57
58impl fmt::Display for ReplaceTimeZoneOptions {
59 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
60 match &self.time_zone {
61 Some(tz) => write!(f, "tz={tz}")?,
62 None => write!(f, "tz=none")?,
63 }
64 write!(f, ", null_on_non_existent={}", self.null_on_non_existent)
65 }
66}
67
68#[derive(Clone)]
72pub struct ReplaceTimeZone;
73
74impl ScalarFnVTable for ReplaceTimeZone {
75 type Options = ReplaceTimeZoneOptions;
76
77 fn id(&self) -> ScalarFnId {
78 static ID: CachedId = CachedId::new("vortex.replace_time_zone");
79 *ID
80 }
81
82 fn serialize(&self, options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
83 Ok(Some(
84 pb::ReplaceTimeZoneOpts {
85 time_zone: options.time_zone.as_ref().map(ToString::to_string),
86 null_on_non_existent: options.null_on_non_existent,
87 }
88 .encode_to_vec(),
89 ))
90 }
91
92 fn deserialize(
93 &self,
94 metadata: &[u8],
95 _session: &VortexSession,
96 ) -> VortexResult<Self::Options> {
97 let options = pb::ReplaceTimeZoneOpts::decode(metadata)?;
98 Ok(ReplaceTimeZoneOptions {
99 time_zone: options.time_zone.map(Arc::from),
100 null_on_non_existent: options.null_on_non_existent,
101 })
102 }
103
104 fn arity(&self, _options: &Self::Options) -> Arity {
105 Arity::Exact(2)
106 }
107
108 fn child_name(&self, _options: &Self::Options, child_idx: usize) -> ChildName {
109 match child_idx {
110 0 => ChildName::from("input"),
111 1 => ChildName::from("ambiguous"),
112 _ => unreachable!("Invalid child index {child_idx} for replace_time_zone()"),
113 }
114 }
115
116 fn simplify(
117 &self,
118 options: &Self::Options,
119 expr: &BoundExpression,
120 ) -> VortexResult<Option<BoundExpression>> {
121 let (Some(value), Some(policy)) = (
122 expr.child(0).as_opt::<Literal>(),
123 expr.child(1).as_opt::<Literal>(),
124 ) else {
125 return Ok(None);
126 };
127 Ok(replace_scalar(options, value, policy).ok().map(bound::lit))
130 }
131
132 fn return_dtype(&self, options: &Self::Options, arg_dtypes: &[DType]) -> VortexResult<DType> {
133 let input = timestamp_options(&arg_dtypes[0])?;
134 if input.unit == TimeUnit::Days {
135 vortex_bail!("Timestamp cannot use day units");
136 }
137 if !matches!(arg_dtypes[1], DType::Utf8(_) | DType::Null) {
138 vortex_bail!(
139 "replace_time_zone() requires a UTF-8 ambiguity policy, got {}",
140 arg_dtypes[1]
141 );
142 }
143 resolve_zone(options.time_zone.as_deref())?;
144 Ok(DType::Extension(
146 Timestamp::new_with_tz(input.unit, options.time_zone.clone(), Nullability::Nullable)
147 .erased(),
148 ))
149 }
150
151 fn execute(
152 &self,
153 options: &Self::Options,
154 args: &dyn ExecutionArgs,
155 ctx: &mut ExecutionCtx,
156 ) -> VortexResult<ArrayRef> {
157 let input = args.get(0)?;
158 let ambiguous = args.get(1)?;
159 let dtype =
160 self.return_dtype(options, &[input.dtype().clone(), ambiguous.dtype().clone()])?;
161 let metadata = timestamp_options(input.dtype())?.clone();
162 let source = resolve_zone(metadata.tz.as_deref())?;
163 let target = resolve_zone(options.time_zone.as_deref())?;
164 let convert = |value, policy: &str| {
165 replace(
166 value,
167 metadata.unit,
168 &source,
169 &target,
170 policy,
171 options.null_on_non_existent,
172 )
173 };
174
175 if let (Some(value), Some(policy)) = (input.as_constant(), ambiguous.as_constant()) {
176 let scalar = replace_scalar(options, value, policy)?;
177 return Ok(ConstantArray::new(scalar, args.row_count()).into_array());
178 }
179
180 if matches!(ambiguous.dtype(), DType::Null) {
181 return Ok(ConstantArray::new(Scalar::null(dtype), args.row_count()).into_array());
182 }
183 let input = input.execute::<ExtensionArray>(ctx)?;
184 let storage = input
185 .storage_array()
186 .clone()
187 .execute::<PrimitiveArray>(ctx)?;
188 let values = storage.to_buffer::<i64>();
189 let valid = storage.validity()?.execute_mask(values.len(), ctx)?;
190 let policies = ambiguous.execute::<VarBinViewArray>(ctx)?;
191 let policies_valid = policies.validity()?.execute_mask(values.len(), ctx)?;
192 let mut output = BufferMut::<i64>::with_capacity(values.len());
193 let mut output_valid = BitBufferMut::with_capacity(values.len());
194 for (i, ((value, valid), policy_valid)) in values
195 .iter()
196 .zip(valid.iter())
197 .zip(policies_valid.iter())
198 .enumerate()
199 {
200 let replaced = if valid && policy_valid {
201 let bytes = policies.bytes_at(i);
202 let policy = unsafe { std::str::from_utf8_unchecked(&bytes) };
204 convert(*value, policy)?
205 } else {
206 None
207 };
208 output.push(replaced.unwrap_or_default());
209 output_valid.append(replaced.is_some());
210 }
211 let DType::Extension(ext) = dtype else {
212 unreachable!()
213 };
214 let storage = PrimitiveArray::new(
215 output.freeze(),
216 Validity::from_bit_buffer(output_valid.freeze(), Nullability::Nullable),
217 )
218 .into_array();
219 Ok(ExtensionArray::new(ext, storage).into_array())
220 }
221}
222
223fn replace_scalar(
225 options: &ReplaceTimeZoneOptions,
226 value: &Scalar,
227 policy: &Scalar,
228) -> VortexResult<Scalar> {
229 let DType::Extension(ext) =
230 ReplaceTimeZone.return_dtype(options, &[value.dtype().clone(), policy.dtype().clone()])?
231 else {
232 unreachable!("replace_time_zone() returns a timestamp")
233 };
234 let metadata = timestamp_options(value.dtype())?;
235 let result = if value.is_null() || policy.is_null() {
236 None
237 } else {
238 let storage = value.as_extension().to_storage_scalar();
239 let policy = policy
240 .as_utf8()
241 .value()
242 .ok_or_else(|| vortex_err!("Missing ambiguity policy"))?;
243 replace(
244 i64::try_from(&storage)?,
245 metadata.unit,
246 &resolve_zone(metadata.tz.as_deref())?,
247 &resolve_zone(options.time_zone.as_deref())?,
248 policy.as_str(),
249 options.null_on_non_existent,
250 )?
251 };
252 let storage = result.map_or_else(
253 || Scalar::null(DType::Primitive(PType::I64, Nullability::Nullable)),
254 |v| Scalar::primitive(v, Nullability::Nullable),
255 );
256 Scalar::try_extension_ref(ext, storage)
257}
258
259fn timestamp_options(dtype: &DType) -> VortexResult<&TimestampOptions> {
260 if let DType::Extension(ext) = dtype
261 && let Some(options) = ext.metadata_opt::<Timestamp>()
262 {
263 return Ok(options);
264 }
265 vortex_bail!("replace_time_zone() requires Timestamp, got {dtype}")
266}
267
268fn replace(
269 value: i64,
270 unit: TimeUnit,
271 source: &TimeZone,
272 target: &TimeZone,
273 policy: &str,
274 null_on_non_existent: bool,
275) -> VortexResult<Option<i64>> {
276 if !matches!(policy, "raise" | "earliest" | "latest" | "null") {
277 vortex_bail!(
278 "Invalid ambiguity policy {policy:?}: expected raise, earliest, latest, or null"
279 );
280 }
281 let timestamp = match unit {
283 TimeUnit::Seconds => JiffTimestamp::from_second(value),
284 TimeUnit::Milliseconds => JiffTimestamp::from_millisecond(value),
285 TimeUnit::Microseconds => JiffTimestamp::from_microsecond(value),
286 TimeUnit::Nanoseconds => JiffTimestamp::from_nanosecond(i128::from(value)),
287 TimeUnit::Days => vortex_bail!("Timestamp cannot use day units"),
288 }
289 .map_err(|error| {
290 VortexError::from(error)
291 .with_context(format!("replace_time_zone() cannot convert {value}{unit}"))
292 })?;
293 if source == target && policy == "raise" {
295 return Ok(Some(value));
296 }
297 let wall_time = source.to_datetime(timestamp);
298 let ambiguous = target.to_ambiguous_timestamp(wall_time);
299 let timestamp = match ambiguous.offset() {
300 AmbiguousOffset::Unambiguous { .. } => ambiguous.unambiguous()?,
301 AmbiguousOffset::Gap { .. } if null_on_non_existent => return Ok(None),
302 AmbiguousOffset::Gap { .. } => ambiguous.unambiguous()?,
303 AmbiguousOffset::Fold { .. } => match policy {
304 "earliest" => ambiguous.earlier()?,
305 "latest" => ambiguous.later()?,
306 "null" => return Ok(None),
307 _ => ambiguous.unambiguous()?,
308 },
309 };
310 Ok(Some(match unit {
311 TimeUnit::Seconds => timestamp.as_second(),
312 TimeUnit::Milliseconds => timestamp.as_millisecond(),
313 TimeUnit::Microseconds => timestamp.as_microsecond(),
314 TimeUnit::Nanoseconds => i64::try_from(timestamp.as_nanosecond())
315 .map_err(|e| vortex_err!("Timestamp overflow: {e}"))?,
316 TimeUnit::Days => unreachable!(),
317 }))
318}
319
320#[cfg(test)]
321mod tests {
322 use std::error::Error;
323 use std::sync::LazyLock;
324
325 use rstest::rstest;
326 use vortex_error::VortexResult;
327 use vortex_error::vortex_err;
328 use vortex_session::VortexSession;
329
330 use super::ReplaceTimeZone;
331 use super::ReplaceTimeZoneOptions;
332 use super::replace;
333 use super::resolve_zone;
334 use crate::Canonical;
335 use crate::IntoArray;
336 use crate::VortexSessionExecute;
337 use crate::arrays::ExtensionArray;
338 use crate::arrays::PrimitiveArray;
339 use crate::arrays::StructArray;
340 use crate::arrays::VarBinViewArray;
341 use crate::assert_arrays_eq;
342 use crate::dtype::DType;
343 use crate::dtype::Nullability;
344 use crate::dtype::StructFields;
345 use crate::expr::get_item;
346 use crate::expr::lit;
347 use crate::expr::replace_time_zone;
348 use crate::expr::root;
349 use crate::extension::datetime::TimeUnit;
350 use crate::extension::datetime::Timestamp;
351 use crate::scalar::Scalar;
352 use crate::scalar_fn::ScalarFnVTable;
353 use crate::scalar_fn::fns::literal::Literal;
354
355 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
356
357 #[rstest]
358 #[case(TimeUnit::Seconds, 1)]
359 #[case(TimeUnit::Milliseconds, 1_000)]
360 #[case(TimeUnit::Microseconds, 1_000_000)]
361 #[case(TimeUnit::Nanoseconds, 1_000_000_000)]
362 fn preserves_units_and_fractional_values(
363 #[case] unit: TimeUnit,
364 #[case] scale: i64,
365 ) -> VortexResult<()> {
366 let utc = resolve_zone(None)?;
367 let new_york = resolve_zone(Some("America/New_York"))?;
368 let value = 1_705_320_000 * scale + scale / 2;
369 assert_eq!(
370 replace(value, unit, &utc, &new_york, "raise", false)?,
371 Some(value + 18_000 * scale)
372 );
373 assert_eq!(
374 replace(
375 value + 18_000 * scale,
376 unit,
377 &new_york,
378 &utc,
379 "raise",
380 false
381 )?,
382 Some(value)
383 );
384 Ok(())
385 }
386
387 #[test]
388 fn resolves_folds_and_gaps_separately() -> VortexResult<()> {
389 let utc = resolve_zone(None)?;
390 let new_york = resolve_zone(Some("America/New_York"))?;
391 let fold = "2024-11-03T01:30:00Z"
392 .parse::<jiff::Timestamp>()?
393 .as_second();
394 let gap = "2024-03-10T02:30:00Z"
395 .parse::<jiff::Timestamp>()?
396 .as_second();
397 let convert = |value, policy, null_on_non_existent| {
398 replace(
399 value,
400 TimeUnit::Seconds,
401 &utc,
402 &new_york,
403 policy,
404 null_on_non_existent,
405 )
406 };
407 assert_eq!(convert(fold, "earliest", false)?, Some(fold + 14_400));
408 assert_eq!(convert(fold, "latest", false)?, Some(fold + 18_000));
409 assert_eq!(convert(fold, "null", false)?, None);
410 assert!(convert(fold, "raise", false).is_err());
411 assert_eq!(convert(gap, "raise", true)?, None);
412 assert!(convert(gap, "earliest", false).is_err());
414 Ok(())
415 }
416
417 #[test]
418 fn rejects_invalid_policy_and_timestamp_overflow() -> VortexResult<()> {
419 let utc = resolve_zone(None)?;
420 assert!(replace(0, TimeUnit::Seconds, &utc, &utc, "invalid", false).is_err());
421 let new_york = resolve_zone(Some("America/New_York"))?;
422 assert!(
423 replace(
424 i64::MAX,
425 TimeUnit::Nanoseconds,
426 &utc,
427 &new_york,
428 "raise",
429 false
430 )
431 .is_err()
432 );
433 assert!(resolve_zone(Some("Not/A/Timezone")).is_err());
434 assert!(resolve_zone(Some("+99:00")).is_err());
435 assert!(resolve_zone(Some("-05:60")).is_err());
436 Ok(())
437 }
438
439 #[rstest]
440 #[case(None, false)]
441 #[case(Some("America/New_York"), true)]
442 fn options_round_trip(
443 #[case] time_zone: Option<&str>,
444 #[case] null_on_non_existent: bool,
445 ) -> VortexResult<()> {
446 let options = ReplaceTimeZoneOptions {
447 time_zone: time_zone.map(Into::into),
448 null_on_non_existent,
449 };
450 let metadata = ReplaceTimeZone.serialize(&options)?.unwrap();
451 assert_eq!(ReplaceTimeZone.deserialize(&metadata, &SESSION)?, options);
452 Ok(())
453 }
454
455 #[rstest]
456 #[case(None, Some("UTC"), 0)]
457 #[case(Some("+01:00"), None, 3_600)]
458 #[case(Some("-05:30"), None, -19_800)]
459 #[case(None, Some("+05:30"), -19_800)]
460 #[case(Some("+01:00"), Some("-05:30"), 23_400)]
461 fn simplify_folds_literal_input(
462 #[case] source: Option<&str>,
463 #[case] target: Option<&str>,
464 #[case] offset_seconds: i64,
465 ) -> VortexResult<()> {
466 let timestamp = |tz: Option<&str>| {
467 Timestamp::new_with_tz(
468 TimeUnit::Microseconds,
469 tz.map(Into::into),
470 Nullability::Nullable,
471 )
472 .erased()
473 };
474 let micros = Scalar::primitive(1_704_153_600_000_000i64, Nullability::Nullable);
475 let expr = replace_time_zone(
476 lit(Scalar::extension_ref(timestamp(source), micros)),
477 lit("earliest"),
478 ReplaceTimeZoneOptions {
479 time_zone: target.map(Into::into),
480 null_on_non_existent: false,
481 },
482 );
483 let scope = DType::Struct(StructFields::empty(), Nullability::NonNullable);
484 let optimized = expr.bind(&scope)?.optimize()?;
485
486 let scalar = optimized
488 .as_opt::<Literal>()
489 .ok_or_else(|| vortex_err!("expected a bare literal, got {optimized}"))?;
490 assert_eq!(
491 scalar,
492 &Scalar::extension_ref(
493 timestamp(target),
494 Scalar::primitive(
495 1_704_153_600_000_000i64 + offset_seconds * 1_000_000,
496 Nullability::Nullable,
497 ),
498 )
499 );
500 assert_eq!(scalar.dtype(), &DType::Extension(timestamp(target)));
501 Ok(())
502 }
503
504 #[test]
505 fn applies_per_row_policies_and_nulls() -> VortexResult<()> {
506 let timestamp = |tz: Option<&str>| {
507 Timestamp::new_with_tz(
508 TimeUnit::Microseconds,
509 tz.map(Into::into),
510 Nullability::Nullable,
511 )
512 .erased()
513 };
514 let fold = 1_730_597_400_000_000i64;
516 let hour = 3_600_000_000i64;
517 let input = ExtensionArray::new(
518 timestamp(None),
519 PrimitiveArray::from_option_iter([
520 Some(fold),
521 Some(fold),
522 Some(fold),
523 Some(fold),
524 None,
525 ])
526 .into_array(),
527 );
528 let policies = VarBinViewArray::from_iter_nullable_str([
529 Some("earliest"),
530 Some("latest"),
531 Some("null"),
532 None,
533 Some("raise"),
534 ]);
535 let array =
536 StructArray::from_fields(&[("t", input.into_array()), ("p", policies.into_array())])?
537 .into_array();
538 let result = array.apply(&replace_time_zone(
539 get_item("t", root()),
540 get_item("p", root()),
541 ReplaceTimeZoneOptions {
542 time_zone: Some("America/New_York".into()),
543 null_on_non_existent: false,
544 },
545 ))?;
546
547 let expected = ExtensionArray::new(
548 timestamp(Some("America/New_York")),
549 PrimitiveArray::from_option_iter([
550 Some(fold + 4 * hour),
551 Some(fold + 5 * hour),
552 None,
553 None,
554 None,
555 ])
556 .into_array(),
557 );
558 let mut ctx = SESSION.create_execution_ctx();
559 assert_arrays_eq!(result, expected, &mut ctx);
560 Ok(())
561 }
562
563 #[rstest]
564 #[case::end_of_9999(TimeUnit::Microseconds, 253_402_214_400_000_000)]
565 #[case::storage_min(TimeUnit::Seconds, i64::MIN)]
566 fn rejects_timestamps_beyond_jiff_range(
567 #[case] unit: TimeUnit,
568 #[case] value: i64,
569 #[values(None, Some("America/New_York"))] target: Option<&str>,
570 ) -> VortexResult<()> {
571 let input = ExtensionArray::new(
572 Timestamp::new(unit, Nullability::Nullable).erased(),
573 PrimitiveArray::from_option_iter([Some(0i64), Some(value)]).into_array(),
574 );
575 let expr = replace_time_zone(
576 root(),
577 lit("raise"),
578 ReplaceTimeZoneOptions {
579 time_zone: target.map(Into::into),
580 null_on_non_existent: false,
581 },
582 );
583 let mut ctx = SESSION.create_execution_ctx();
584 let result = input
585 .into_array()
586 .apply(&expr)
587 .and_then(|array| array.execute::<Canonical>(&mut ctx));
588 let error = result.unwrap_err();
589 assert!(error.to_string().contains("cannot convert"), "{error}");
590 assert!(
591 error
592 .source()
593 .is_some_and(|source| source.is::<jiff::Error>())
594 );
595
596 let control = ExtensionArray::new(
597 Timestamp::new(unit, Nullability::Nullable).erased(),
598 PrimitiveArray::from_option_iter([Some(0i64)]).into_array(),
599 );
600 control
601 .into_array()
602 .apply(&expr)?
603 .execute::<Canonical>(&mut ctx)?;
604
605 Ok(())
606 }
607}